# -*- coding: utf-8 -*-

import os
import time
import json
import codecs
import requests
from multiprocessing import Process,Queue
import gevent
from gevent import monkey,pool
# monkey.patch_all()

from entrance import *
#for slave local
from config import ENTER_1,ENTER_2,ENTER_3, TIMEOUT 
#for dispatch task
from config import (MASTER_IP, MASTER_PORT, LOOP_TIME, MAX_TASK,
                GET_TASKS_AMOUT, TASK_POOL,SECRET)



def back_taskids(Q_FINISH,Q_FAILED):
    """feedback task`s status to master"""

    if Q_FINISH.qsize() < 500:
        return
    finish = [Q_FINISH.get() for x in xrange(500)]
    if Q_FAILED.qsize() > 0:
        failed = [Q_FAILED.get() for x in xrange(Q_FAILED.qsize())]
    else:
        failed = []
    try:
        headers = {'secret':SECRET}
        taskids = {'finish':finish,'failed':failed}
        url = 'http://'+MASTER_IP+':'+MASTER_PORT+'/back_taskids'
        r = requests.get(url, headers=headers,json=taskids)
        if r.status_code == 200 and r.json()['msg'] == 'success':
            print('success back taskid to master...')
    except Exception as e:
        print(e)


def get_tasks(Q_TASK):
    """func get tasks from master and put task to pool"""

    if GET_TASKS_AMOUT > TASK_POOL - Q_TASK.qsize():
        print ('tasks pool full,wait a minute....')
        return 
    try:
        url = 'http://'+MASTER_IP+':'+MASTER_PORT+'/get_tasks'
        headers = {'secret':SECRET,'count':str(GET_TASKS_AMOUT)}
        r = requests.get(url, headers=headers, timeout=20)
        tasks = r.json() if r.status_code == 200 else None
    except Exception as e:
        print(e)
        return
    tasks = tasks.get('tasks','')
    if isinstance(tasks,list):
        map(lambda task:Q_TASK.put(task),tasks)


def loop_system(Q_TASK,Q_FINISH,Q_FAILED):
    """loop system to ensure tasks system"""
    while True:
        get_tasks(Q_TASK)
        back_taskids(Q_FINISH,Q_FAILED)
        print ('tasks pool size is {}'.format(Q_TASK.qsize()))
        time.sleep(LOOP_TIME)


def doing(task,Q_FINISH,Q_FAILED):
    """func for query"""
    if not isinstance(task,dict):
        return
    task_id = task['task_id']
    try:
        entrance =  task['entrance']
        task['person'] = eval(task['person'])

        g_tasks =[]
        if '1' in entrance:
            for m in ENTER_1:
                func = eval(m+'.query_wrap')
                g_tasks.append(gevent.spawn(func,task,TIMEOUT))  
        if '2' in entrance:
            for m in ENTER_2:
                func = eval(m+'.query_wrap')
                g_tasks.append(gevent.spawn(func,task,TIMEOUT))            
        if '3' in entrance:
            for m in ENTER_3:
                func = eval(m+'.query_wrap')
                g_tasks.append(gevent.spawn(func,task,TIMEOUT))

        gevent.joinall(g_tasks,timeout=10*TIMEOUT)
        Q_FINISH.put(task_id)
    except Exception as e:
        print(e)
        Q_FAILED.put(task_id)


def dispatch():

    #任务池队列
    Q_TASK = Queue(-1)
    #向mater回馈成功的taskid队列
    Q_FINISH = Queue(-1)
    #向master回馈失败的taskid队列
    Q_FAILED = Queue(-1)
    #爬取并发数量
    # POOL = pool.Pool(MAX_TASK)
    POOL = pool.Pool(3)

    p = Process(target=loop_system,args=(Q_TASK,Q_FINISH,Q_FAILED))
    p.start()
    print('start a process-{} get tasks...'.format(p.pid))

    while  True:
        if Q_TASK.qsize():
            task = Q_TASK.get()
            POOL.spawn(doing,task,Q_FINISH,Q_FAILED)
        else:
            time.sleep(5)
            print('Q_TASK is empty,nothing to do ...')
        time.sleep(0.01)


if __name__ == '__main__':
    dispatch()
        
 